import facade
import os
import sys
import time

from xjlibrary.our_file_dir import BaseDir

bChange = False
StartTime = time.time()
curPath = os.path.dirname(os.path.abspath(__file__))
HEADERS = {'Accept': '*/*', 'Accept-Encoding': 'gzip, deflate', 'Connection': 'keep-alive',
           'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C; .NET4.0E)', }
sSqlList = list()
proxiesList = list()
nCount = 0
Proxies = {
    'http': '192.168.30.4:8080',
    'https': '192.168.30.4:8080'  # key是指目标网站的协议
}

mysqlutils = facade.MysqlUtiles(BaseDir.get_new_path(curPath, "db.ini"), "db", facade.get_streamlogger())


# 测试文字输出到文本
def output(value, files='log.txt'):
    f = open(files, 'w', encoding='utf-8')
    f.write(value)
    f.close()


# 获取代理词
def GetProxies():
    sProxSql = 'select * from proxy_pool'
    rows = mysqlutils.SelectFromDB(sProxSql)
    return rows


# 从代理池中取一个代理
def GetProx():
    global proxiesList

    # bChange = False
    if len(proxiesList) < 3:
        lst = GetProxies()
        proxiesList = list(set(proxiesList) | set(lst))

    prox = proxiesList[0]
    del proxiesList[0]
    tmpDict = dict()
    tmpDict['http'] = prox[0]
    tmpDict['https'] = prox[0]
    prox = tmpDict

    return prox


# 获取详情页列表
def GetHtml(nNum):
    sSql = "SELECT filename FROM article WHERE stat = 0 AND failcount < 3  LIMIT {nNum}".format(nNum=nNum)
    rows = mysqlutils.SelectFromDB(sSql)
    return rows


# 下载详情页面
def DownPage(taskList):
    global sSqlList, nCount
    for task in taskList:
        if not task:
            print('* Task Error')
            sys.exit(-1)

        filename = task[0]
        profix = filename[:4]
        # 文件目录名称
        pageUrl = 'http://dbpub.cnki.net/grid2008/dbpub/detail.aspx?dbname=%s&filename=%s' % (profix, filename)
        BoolResult, errString, r = facade.BaseRequest(pageUrl,
                                                      proxies=Proxies,
                                                      headers=HEADERS,
                                                      mark="【标准号】",
                                                      timeout=(20, 30))
        if BoolResult:
            if r.text.find(filename) == -1:
                sSql = "UPDATE article set failcount = failcount + 1 where filename = '{filename}'".format(
                    filename=filename)
            else:
                try:
                    html = r.content.decode('utf-8').strip()
                except:
                    print('cant decode by utf-8')
                    sSql = "UPDATE article set failcount = failcount + 1 where filename = '{filename}'".format(
                        filename=filename)
                    sSqlList.append(sSql)
                    UpdateDbById()
                    continue
                line = filename + '★' + html.replace('\0', ' ').replace('\r', ' ').replace('\n', ' ') + '\n'
                outPathFile = os.path.join(curPath, 'cnki_bz_append.big_htm')
                with open(outPathFile, mode='a', encoding='utf-8') as f:
                    f.write(line)
                    nCount += 1
                    print('写入文件数量：%d' % nCount)
                    print('耗费时间: %d' % int(time.time() - StartTime) + '秒')

                sSql = "UPDATE article set stat = 1 where filename = '{filename}'".format(filename=filename)
        else:
            try:
                if r.text.find("未找到该条记录") > -1:
                    sSql = "UPDATE article set stat = -1 where filename = '{filename}'".format(
                        filename=filename)
                else:
                    sSql = "UPDATE article set failcount = failcount + 1 where filename = '{filename}'".format(
                        filename=filename)
            except:
                sSql = "UPDATE article set failcount = failcount + 1 where filename = '{filename}'".format(
                    filename=filename)
        sSqlList.append(sSql)
    UpdateDbById()


# 更新数据库状态
def UpdateDbById():
    global sSqlList
    mysqlutils.ExeSqlListToDB(sSqlList)
    sSqlList = list()


# ----------------------------------------------逻辑部分-----------------------------------------------
# ----------------------------------------------下载详情页---------------------------------------------

def main(logger=None):
    while True:
        taskList = GetHtml(1000)
        if not taskList:
            break
        DownPage(taskList)


if __name__ == '__main__':
    main()
